# encoding: utf-8
from datetime import datetime
import time
import traceback
from multiprocessing.pool import ThreadPool
from com.util.dateUtil import tradedays

import bs4
from bs4 import BeautifulSoup

from com.util.httpRequest import HttpRequest
from com.util.pushUtil import pushDingDing, pushDingDingToWogoo
from com.util.loggingUtil import Logs
from com.util.mysqlPool import MysqlConnection

Log = Logs()
# 对比净值数 近1年数据
limit = 250
successMap = {}
failMap = {}
# FundCode映射到MainCode
Fund_MainCode = {}


def getAllFunds():
    """
    获取所有场外基金code
    :return:
    """
    sql =  f"select  distinct FundCode, MainCode from MF_FundMain where " \
           f"isEnd = 0  and UnderlyingClassify in ('股票型','债券型','混合型')"
    list = MysqlConnection('big-data-test').query(sql)
    returnList = []
    for obj in list:
        # 处理数据对应的MainCode 对应MainCode的对应个数
        Fund_MainCode.setdefault(obj['FundCode'], obj['MainCode'])
        returnList.append(obj['FundCode'])
    return returnList


def getBigDataFundValue(fundCode):
    """
    获取基金净值
    :param FundCode:
    :return:
    """
    sql = f"select date_format(Date,'%Y%m%d') as date,UnitNV,AccumulatedUnitNV " \
          f"from MF_FundNetvalue where FundCode = '{fundCode}' ORDER BY date desc limit {limit}"
    list = MysqlConnection('big-data-test').query(sql)
    returnList = []
    for obj in list:
        singleData = {
            'date': obj['date'],
            # None转成了字符串'None'
            'dwjz': str(obj['UnitNV']),
            # None转成了字符串'None'
            'ljjz': str(obj['AccumulatedUnitNV'])
        }
        returnList.append(singleData)
    return returnList


def getDCFundValue(fundCode):
    """
    东方财富基金
    :param fundCode:
    :return:
    """
    url = f"http://api.fund.eastmoney.com/f10/lsjz?" \
          f"fundCode={fundCode}&pageIndex=1&pageSize={limit}&startDate=&" \
          f"endDate=&_=1654667994665"
    cookie_headers = {
        'Referer': 'http://fundf10.eastmoney.com/',
    }
    jsonRes = HttpRequest().http_request(method='GET', url=url, headers=cookie_headers)
    datalist = jsonRes['Data']['LSJZList']
    returnRes = {}
    for data in datalist:
        date = str(data['FSRQ']).replace('-', '')
        singleData = {
            'dwjz': data['DWJZ'],
            'ljjz': data['LJJZ']
        }
        returnRes.setdefault(date, singleData)
    return returnRes


def getDC_APPFundValue(fundCode):
    time.sleep(1)
    """
    东财App
    :param fundCode:
    :return:
    """
    url = f"https://datacenter.eastmoney.com/securities/api/data/get?" \
          f"filter=(SECUCODE%3D%22{fundCode}.OF%22)&client=APP&source=HSF10&" \
          f"type=RPT_F10_FUND_PERNAV&sty=SECURITY_CODE,SECURITY_INNER_CODE,SECUCODE,SECURITY_NAME,SECURITY_NAME_ABBR,END_DATE,PER_NAV,PER_ACCUM_NAV,ADJNAV_GR,INCOME_TEN_THOUSAND,YIELD_7DAYS,FUND_TYPE,FUND_TYPE_CODE,SPECIAL_MARK,THOUSAND_TYPE,IS_LARGE_REDEEM" \
          f"&ps={limit}&sr=-1&p=1&st=END_DATE"

    jsonRes = HttpRequest().http_request(method='GET', url=url)
    datalist = jsonRes['result']['data']
    returnRes = dict()
    for data in datalist:
        date = str(data['END_DATE'])[0:10].replace('-', '')
        singleData = {
            'dwjz': data['PER_NAV'],
            'ljjz': data['PER_ACCUM_NAV']
        }
        returnRes.setdefault(date, singleData)
    return returnRes


def CompareFund(dbResult, httpData, fundCode):
    """
   比较分时数据
   :param dbResult:  mongodb数据
   :param httpRes:  抓取的数据
   :param typeName: 比较的数据
   :return:
   """
    mysqlNum = 0
    httpNum = 0
    failNum = 0
    errorList = []
    for obj in dbResult:  # 遍历数据
        mysqlNum = mysqlNum + 1
        date = obj['date']
        # 跳过错误的数据
        key = f"{fundCode}-{date}"

        hData: dict = httpData.get(date)
        if hData:
            httpNum = httpNum + 1
            pass
            # Log.info('校验日期' + str(market_date))
        else:  # 第三方数据不存在直接跳过
            continue

        errorData = {"date": date}
        # 单位净值
        dwjz = obj['dwjz']
        dwjz = '0.00' if dwjz == 'None' else dwjz
        http_dwjz = hData.get('dwjz')
        http_dwjz = '0.00' if http_dwjz == '' else http_dwjz
        # 跳过我们是None的，东财是1.0000的
        if not (obj['dwjz'] == 'None' and http_dwjz == '1.0000') and hData.get('dwjz') != '':
            if float(dwjz) != float(http_dwjz):
                errorData.setdefault("dwjz", f"{obj['dwjz']} != {http_dwjz}")
        # 累计净值
        ljjz = obj['ljjz']
        ljjz = '0.00' if ljjz == 'None' else ljjz
        http_ljjz = hData.get('ljjz')
        http_ljjz = '0.00' if http_ljjz == '' else http_ljjz
        # 跳过我们是None的，东财是1.0000的
        if not (obj['ljjz'] == 'None' and http_ljjz == '1.0000') and hData.get('ljjz') != '':
            if float(ljjz) != float(http_ljjz):
                errorData.setdefault('ljjz', f"{obj['ljjz']} != {http_ljjz}")

        if errorData.__len__() > 1:
            errorList.append(errorData)
            failNum = failNum + 1
            Log.error(f"code:[{fundCode}]校验日期{str(date)}"
                      f"errData:[{errorData}]")

    Log.info(f"code:[{fundCode}]校验完毕mysql总数:{mysqlNum},httpData总数{httpNum},校验数{httpNum}错误数{failNum}")
    if failNum == 0:
        successMap.setdefault(fundCode, {'compareNum': httpNum})
    else:
        failMap.setdefault(fundCode, {'compareNum': httpNum, 'failNum': failNum, "errors": errorList})


# 防止死循环
fund_while = {}


def getZBMarket(mainCode, fundCode, classification, needDate, start):
    """
    资本市场信息披露话平台
    :param fundCode:
    :return:
    """
    # 防止爬取太快导致封ip
    time.sleep(0.2)
    # 防止死循环
    key = f"{fundCode}-{needDate}"
    if fund_while.__contains__(key):
        value = int(fund_while[key])
        if value >= 6:
            return None
        else:
            fund_while.__setitem__(key, value + 1)
    else:
        fund_while.setdefault(key, 1)

    if start < 0:
        start = 0

    if classification:
        url = f"http://eid.csrc.gov.cn/fund/web/list_net_classification.daily_report?" \
              f"1=1&fundCode={mainCode}&classification={classification}&limit=20&start={start}"
    else:
        url = f"http://eid.csrc.gov.cn/fund/web/list_net_classification.daily_report?" \
              f"1=1&fundCode={mainCode}&limit=20&start={start}"

    Log.info(f"请求数据url:{url}")
    header = {
        "Content-Type": "text/html;charset=UTF-8",
        "Cookie": "acw_tc=7ce1a71c16652943817675015e094ba329d182807eadbc8e57862c4e4c",
        "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/105.0.0.0 Safari/537.36"
    }
    htmlStr = HttpRequest().http_request(method='GET', url=url, headers=header)
    resDict = parseZBMarketHtml(htmlStr, fundCode, classification)

    # 没有结果回传False
    if resDict.__len__() == 0:
        return None

    try:
        total = int(resDict['total'])
    except:
        # total ="没有数据时"
        return None

    if resDict.__len__() == 2:
        return getZBMarket(mainCode, fundCode, classification, needDate, total - 10)
    maxCount = resDict['maxCount']
    minDate = resDict['minDate']
    maxDate = resDict['maxDate']
    Log.info(f"资本市场结果fundCode:{fundCode}=minDate:{minDate},maxDate:{maxDate},total:{total}")

    nd = needDate[0:4] + '-' + needDate[4:6] + '-' + needDate[6:8]
    if needDate < minDate:
        maxd = maxDate[0:4] + '-' + maxDate[4:6] + '-' + maxDate[6:8]
        count = tradedays(nd, maxd)
        return getZBMarket(mainCode, fundCode, classification, needDate, start + count * maxCount - 3 * maxCount)
    if needDate > maxDate:
        minD = minDate[0:4] + '-' + minDate[4:6] + '-' + minDate[6:8]
        count = tradedays(minD, nd)
        num = start - count * maxCount + 3 * maxCount
        if num < 0:
            num = int(start / 2)
        return getZBMarket(mainCode, fundCode, classification, needDate, num)
    if minDate <= needDate and needDate <= maxDate:
        Log.info(f"资本市场结果fundCode:{fundCode}==={resDict.get(needDate)}")
        return resDict.get(needDate)


def parseZBMarketHtml(htmlStr, fundCode, classification):
    # 1、BeautifulSoup对象
    soup = BeautifulSoup(htmlStr, 'html.parser')
    result = {}
    trs = soup.select('table>tr.dd,table>tr.aa')

    # 查询条数
    td_last = soup.select('td.Contentbox>div>table td:nth-child(2)')[0].contents[0]
    num_left = int(str(td_last).find('共'))
    num_right = int(str(td_last).find('条'))
    total = td_last[num_left + 1:num_right]
    minDate = ""
    maxDate = ""
    maxCount = 0
    defaultDateCount = {}

    for tr in trs:
        tr: bs4.element.Tag
        tds = tr.select('td')

        date = str(tds[6].contents[0]).replace('-', '')

        if defaultDateCount.__contains__(date):
            val = int(defaultDateCount.get(date))
            defaultDateCount.__setitem__(date, val + 1)
            if val + 1 >= maxCount:
                maxCount = val + 1
        else:
            defaultDateCount.setdefault(date, 1)
            if 1 >= maxCount:
                maxCount = 1
        # 分级代码
        fjdm = tds[1].contents[0]
        # 基金代码即主代码
        jjdm = tds[0].contents[0]
        try:
            dwjz = tds[3].contents[0]
            ljjz = tds[4].contents[0]
        except:
            dwjz = None
            ljjz = None

        # 有分级代码且不一致直接回传
        if classification and str(fjdm).__ne__(fundCode):
            return {}
        # 没有分级代码且code都不一致的直接跳过
        if not classification and str(fjdm).__ne__(fundCode) and str(jjdm).__ne__(fundCode):
            continue

        if (minDate != "" and date < minDate) or minDate == "":
            minDate = date

        if (maxDate != "" and date > maxDate) or maxDate == "":
            maxDate = date

        singleData = {
            'jjdm': jjdm,
            'fjdm': fjdm,
            'date': date,
            'dwjz': dwjz,
            'ljjz': ljjz
        }
        # 有分级代码优先覆盖基金代码
        if str(jjdm).__eq__(fundCode) and not result.__contains__(date):
            result.__setitem__(date, singleData)
        if str(fjdm).__eq__(fundCode):
            result.__setitem__(date, singleData)
    if maxDate != "" and minDate != "":
        result.setdefault('maxDate', maxDate)
        result.setdefault('minDate', minDate)
    result.setdefault('total', total)
    result.setdefault('maxCount', maxCount)
    return result


def multiCompareFund(fundCode):
    """
    对比基金
    :param fundCode:
    :param isDc: 是否和东财比较，否与新浪比较
    :return:
    """

    try:
        mysqlData = getBigDataFundValue(fundCode)
        dcData = getDCFundValue(fundCode)
        Log.info(f"code:[{fundCode}]东财对比数据")
        CompareFund(mysqlData, dcData, fundCode)
    except:
        Log.error(f"-*****************code error [{fundCode}] ******************-{traceback.format_exc()}")
        failMap.setdefault(fundCode, {'httpError': True, 'errors': []})


def toGetClassification(mainCode, fundCode):
    list = MysqlConnection('big-data-test').query(f"select FundCode,RIGHT(FundShortName,1) fundName from MF_FundMain "
                                                  f"where MainCode = '{mainCode}' and IsEnd = 0 ORDER BY fundName ASC")
    Classification_list = ['2030-1010', '2030-1020', '2030-1040',
                           '2030-1050', '2030-1060', '2030-1070', '2030-1080']
    # 没有分级代码就直接返回
    if list.__len__() == 1:
        return None

    num = 0
    for obj in list:
        if str(fundCode).__eq__(obj['FundCode']):
            return Classification_list[num]
        else:
            num = num + 1
    return None


fund_zb_error = {}


def setError(fundCode, data):
    if fund_zb_error.__contains__(fundCode):
        value_list: list = fund_zb_error[fundCode]
        value_list.append(data)
        fund_zb_error.__setitem__(fundCode, value_list)
    else:
        fund_zb_error.__setitem__(fundCode, [data])


def toCompareValue(fundCode, date, dwjz, ljjz):
    mainCode = Fund_MainCode.get(fundCode)
    classification = toGetClassification(mainCode, fundCode)

    # 获取两个日期的交易日个数
    today = datetime.today().date()
    startDate = date[0:4] + '-' + date[4:6] + '-' + date[6:8]
    count = tradedays(startDate, today)

    start = count - 10
    needData = getZBMarket(mainCode, fundCode, classification, date, start)

    # 不存在数据
    if not needData:
        Log.error(f"不存在比较数据fundCode:{fundCode},date:{date},dwjz:{dwjz},ljjz:{ljjz}")
        # setError(fundCode, {"date": date, "dwjz": dwjz, "ljjz": ljjz, "msg": "not find"})
        return False

    errorData = {}

    # 是否返回False
    flag = False
    if dwjz and needData['dwjz']: # 去除非空判定
        if float(dwjz) != float(needData['dwjz']):
            Log.error(f"比较失败fundCode:{fundCode},date:{date},dwjz:{dwjz},ljjz:{ljjz}，\n待比较值needData{needData}")
            errorData.__setitem__("date", date)
            errorData.__setitem__("dwjz", f"{dwjz} != {needData['dwjz']}")
            flag = True
    if ljjz and needData['ljjz']: # 去除非空判定
        if float(ljjz) != float(needData['ljjz']):
            Log.error(f"比较失败fundCode:{fundCode},date:{date},dwjz:{dwjz},ljjz:{ljjz}，\n待比较值needData{needData}")
            errorData.__setitem__("date", date)
            errorData.__setitem__("ljjz", f"{ljjz} != {needData['ljjz']}")
            flag = True
    if flag:
        setError(fundCode, errorData)
        return False
    else:
        Log.info(f"比较成功fundCode:{fundCode},date:{date},dwjz:{dwjz},ljjz:{ljjz}")
        return True


def compareAgain():
    for fundCode in failMap.keys():
        errLists: list = failMap.get(fundCode)['errors']

        final_errLists = []
        for errData in errLists:
            errData: dict
            date = errData['date']
            dwjz = None
            ljjz = None
            if errData.__contains__('dwjz'):
                dwjz = str(errData['dwjz']).split(' ')[0]
                if dwjz.__contains__('None'):
                    dwjz=None
            if errData.__contains__('ljjz'):
                ljjz = str(errData['ljjz']).split(' ')[0]
                if ljjz.__contains__('None'):
                    ljjz=None

            Log.info(f"to-compare,fundCode:{fundCode},date:{date},dwjz:{dwjz},ljjz:{ljjz}")
            # 一样为True 不一样为False
            flag = toCompareValue(fundCode, date, dwjz, ljjz)
            # 比较为假
            if not flag:
                final_errLists.append(errData)

        if final_errLists.__len__() > 0:
            failMap.__setitem__(fundCode, {"errors": final_errLists})
        else:
            # 去除对比后清空的错误数据,底部会进行过滤， 循环里直接删除会出现问题
            failMap.__setitem__(fundCode, {'filter': True})


def run():
    fundCodes = getAllFunds();
    # fundCodes = fundCodes[0:100]
    pool = ThreadPool(5)
    pool.map(multiCompareFund, fundCodes)
    pool.close()
    pool.join()

    totalsize = successMap.__len__() + failMap.__len__()
    resultContent = f"东财-对比基金总数:【{totalsize}】,校验成功:【{successMap.__len__()}】,校验失败【{failMap.__len__()}】。"
    pushContent = f"{resultContent}东财对比错误数据：{failMap}"
    Log.info(pushContent)

    # 拿取错误集合再与资本市场电子信息化平台对比
    compareAgain()

    Log.info(f"failMap:{failMap}")

    # 过滤 httpError的结果
    fail_res = {}
    http_error_num = 0
    for key in failMap.keys():
        value = failMap.get(key)
        if str(value).__contains__('httpError'):
            http_error_num = http_error_num + 1
        elif str(value).__contains__('filter'):
            pass
        else:
            fail_res.setdefault(key, value)

    totalsize = successMap.__len__() + fail_res.__len__()
    successNum = totalsize - fund_zb_error.__len__()
    resultContent = f"to吾股:对比基金净值总数:【{totalsize}】,校验成功:【{successNum}】,校验失败【{fund_zb_error.__len__()}】,抓取失败【{http_error_num}】。"
    pushContent = f"{resultContent}对比错误数据：{fund_zb_error}"
    Log.info(pushContent)
    pushDingDing(pushContent)
    # res = pushDingDingToWogoo(pushContent)
    # Log.info(res)


if __name__ == '__main__':
    run()
